//package com.changxianggu.crawler.cloud.test;
//
//import cn.hutool.core.convert.Convert;
//import cn.hutool.core.io.file.FileReader;
//import cn.hutool.core.map.MapUtil;
//import cn.hutool.core.util.ArrayUtil;
//import cn.hutool.core.util.RandomUtil;
//import cn.hutool.core.util.StrUtil;
//import com.alibaba.fastjson.JSONArray;
//import com.alibaba.fastjson.JSONObject;
//import com.changxianggu.cloud.crawler.core.WebApplication;
//import com.changxianggu.cloud.crawler.core.modules.app.entity.CxgQinOnlineCourse;
//import com.changxianggu.cloud.crawler.core.modules.app.entity.CxgQinOnlineCourseItem;
//import com.changxianggu.cloud.crawler.core.modules.app.service.ICxgQinOnlineCourseItemService;
//import com.changxianggu.cloud.crawler.core.modules.app.service.ICxgQinOnlineCourseService;
//import com.changxianggu.cloud.crawler.core.modules.storage.entity.WyExcelTestVO;
//import com.changxianggu.cloud.crawler.core.utils.HttpUtil;
//import com.google.gson.JsonObject;
//import lombok.extern.slf4j.Slf4j;
//import org.jsoup.Jsoup;
//import org.jsoup.nodes.Document;
//import org.jsoup.nodes.Element;
//import org.jsoup.select.Elements;
//import org.junit.Test;
//import org.junit.runner.RunWith;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.boot.test.context.SpringBootTest;
//import org.springframework.test.context.ActiveProfiles;
//import org.springframework.test.context.junit4.SpringRunner;
//
//import java.lang.reflect.Array;
//import java.math.BigDecimal;
//import java.util.*;
//import java.util.regex.Matcher;
//import java.util.regex.Pattern;
//
//@RunWith(SpringRunner.class)
//@ActiveProfiles("master")
//@SpringBootTest(classes = {WebApplication.class})// 指定启动类
//@Slf4j
//public class Wangyi {
//
//    @Autowired
//    ICxgQinOnlineCourseService courseService;
//    @Autowired
//    ICxgQinOnlineCourseItemService courseItemService;
//
//    public static final String COOKIE = "_ntes_nnid=2ba9c27042ad7c5ab73d485ec917f2b8,1627024350767; _ntes_nuid=2ba9c27042ad7c5ab73d485ec917f2b8; __oc_uuid=584a1580-eb85-11eb-a7f2-2faf5ac727dc; vinfo_n_f_l_n3=bd4c28ae8346d87c.1.0.1627376274839.0.1627376317642; timing_user_id=time_7pYRGwack8; _antanalysis_s_id=1629684661798; NTESOPENSI=8d16e00c47bd48ca908161c93b2aa115";
//    public static final String URL = "https://open.163.com/cuvocw/";
//    public static final String URL_PRE = "https://open.163.com/cuvocw/";
//    public static final String PREFIX = "网易公开课-中国大学视频公开课-";
//
//
//    @Test
//    public void initCrawler() throws Exception {
//
////        String path1 = "F:\\1.txt";
////        String path1 = "F:\\1.txt";
////        String path2 = "F:\\2.txt";
////        String path3 = "F:\\3.txt";
////        String path4 = "F:\\4.txt";
////        String path5 = "F:\\5.txt";
////        String path6 = "F:\\6.txt";
////          String path7 = "F:\\ss\\7.txt";
////          String path8 = "F:\\ss\\8.txt";
////          String path9 = "F:\\ss\\9.txt";
////          String path10 = "F:\\ss\\10.txt";
////          String path11 = "F:\\ss\\11.txt";
////          String path12 = "F:\\ss\\12.txt";
////          String path13 = "F:\\ss\\13.txt";
////          String path14 = "F:\\ss\\14.txt";
////          String path15 = "F:\\ss\\15.txt";
////          String path16 = "F:\\ss\\16.txt";
////          String path17 = "F:\\ss\\17.txt";
////          String path18 = "F:\\ss\\18.txt";
////          String path19 = "F:\\ss\\19.txt";
////          String path20 = "F:\\ss\\20.txt";
////          String path21 = "F:\\ss\\21.txt";
////          String path22 = "F:\\ss\\22.txt";
////          String path23 = "F:\\ss\\23.txt";
////          String path24 = "F:\\ss\\24.txt";
////          String path25 = "F:\\ss\\25.txt";
////          String path26 = "F:\\ss\\26.txt";
////          String path27 = "F:\\ss\\27.txt";
////          String path28 = "F:\\ss\\28.txt";
////          String path29 = "F:\\ss\\29.txt";
//          String path0 = "F:\\ss\\0.txt";
//
//        List<String> pathList = new ArrayList<>();
//        pathList.add(path0 );
////        pathList.add(path7 );
////        pathList.add(path8 );
////        pathList.add(path9 );
////        pathList.add(path10);
////        pathList.add(path11);
////        pathList.add(path12);
////        pathList.add(path13);
////        pathList.add(path14);
////        pathList.add(path15);
////        pathList.add(path16);
////        pathList.add(path17);
////        pathList.add(path18);
////        pathList.add(path19);
////        pathList.add(path20);
////        pathList.add(path21);
////        pathList.add(path22);
////        pathList.add(path23);
////        pathList.add(path24);
////        pathList.add(path25);
////        pathList.add(path26);
////        pathList.add(path27);
////        pathList.add(path28);
////        pathList.add(path29);
//        for (String path : pathList) {
//            doCrawler(getHtmlStr(path));
//        }
//    }
//
//    public void proxy() {
//        Map<String, String> map = new HashMap<>();
//        map.put("proxy1", "http://122.138.38.145:4245");
//        map.put("port1", "4245");
//        map.put("proxy2", "http://124.231.52.223:4235");
//        map.put("port2", "4235");
//        map.put("proxy3", "http://182.136.97.123:4256");
//        map.put("port3", "4256");
//        Integer st = RandomUtil.randomInt(1, 3);
//
//        System.setProperty("https.proxyPort", map.get("port" + st.toString()));
//        System.setProperty("https.proxyHost", map.get("proxy" + st.toString()));
//    }
//
//    public String getHtmlStr(String path) {
//        FileReader file1Reader = new FileReader(path);
//        String result = file1Reader.readString();
//        return result;
//
//    }
//
//    public void doCrawler(String htmlStr) throws Exception {
//        if (StrUtil.isNotBlank(htmlStr)) {
//            Document htmlDoc = Jsoup.parse(htmlStr);
//            Elements bookContentList = htmlDoc.select(".u-cover");
//            List<WyExcelTestVO> dataList = new ArrayList<>();
//            if (bookContentList != null && bookContentList.size() > 0) {
//                for (Element element : bookContentList) {
//                    if (element != null) {
//                        //详情地址
//                        String href = element.attr("href");
//                        if (StrUtil.isNotEmpty(href)) {
//                            String pid = getPid(href);
//                            if (StrUtil.isNotEmpty(pid)) {
//                                WyExcelTestVO entity = getBookEntity(pid, href);
//                                dataList.add(entity);
//                            }
//                        }
//                    }
//                }
//            }
//        }
//
//    }
//
//    public String getPid(String url) throws Exception {
//        String pid = "";
//        if (StrUtil.isBlank(url)) {
//            return null;
//        }
//        Map<String, Object> headers = MapUtil.newHashMap(1);
//        // proxy();
//        String result = HttpUtil.doGet(url, headers);
//        int tryCount = 1;
//        while (result.contains("页面没有找到") && tryCount <= 5) {
//            //睡眠2秒继续访问
//            Thread.sleep(2000);
//            // proxy();
//            result = HttpUtil.doGet(url, headers);
//            tryCount++;
//        }
//        if (StrUtil.isNotBlank(result)) {
//            Document htmlDoc = null;
//            try {
//                htmlDoc = Jsoup.parse(result);
//            } catch (Exception e) {
//                log.warn("jsoup解析html发生错误");
//                log.warn("错误：" + url);
//            }
//            if (htmlDoc == null) {
//                return null;
//            }
//            Elements listLink = htmlDoc.select(".list-link");
//            if (listLink != null && listLink.size() > 0) {
//                String newUrl = listLink.get(0).select("a").attr("href");
//                if (StrUtil.isNotEmpty(newUrl)) {
//                    pid = newUrl.split("pid=")[1].split("&")[0];
//                }
//            }
//        }
//        return pid;
//
//    }
//
//    public WyExcelTestVO getBookEntity(String pid, String detailUrl) throws Exception {
//        WyExcelTestVO bookEntity = null;
//        if (StrUtil.isBlank(pid)) {
//            return null;
//        }
//        String url = "https://c.open.163.com/open/mob/movie/list.do?plid=" + pid;
//        Map<String, Object> headers = MapUtil.newHashMap(1);
//        //睡眠2秒继续访问
//        Thread.sleep(500);
//        String result = HttpUtil.doGet(url, headers);
//        int tryCount = 1;
//        while (result.contains("页面没有找到") && tryCount <= 5) {
//            //睡眠2秒继续访问
//            Thread.sleep(2000);
//            // proxy();
//            result = HttpUtil.doGet(url, headers);
//            tryCount++;
//        }
//        if (StrUtil.isNotBlank(result)) {
//            JSONObject jsonObject = JSONObject.parseObject(result);
//            JSONObject data = jsonObject.getJSONObject("data");
//            if (data != null) {
//                JSONArray listObject = data.getJSONArray("videoList");
//                CxgQinOnlineCourse course = new CxgQinOnlineCourse();
//                course.setTitle(data.get("title").toString());
//                course.setLecturer(data.get("director").toString());
//                course.setCover(data.get("largeImgurl").toString());
//                course.setBrief(data.get("description").toString());
//                course.setLinkUrl(detailUrl);
//                try {
//                    course.setKeywordArr(data.get("classifyName").toString());
//                } catch (Exception e) {
//                }
//                course.setPrice(new BigDecimal("0.00"));
//                course.setIsLink(1);
//                courseService.save(course);
//                int size = listObject.size();
//                List<CxgQinOnlineCourseItem> itemList = new ArrayList<>();
//                for (int k = 0; k < size; k++) {
//                    JSONObject jsonTmp = listObject.getJSONObject(k);
//                    CxgQinOnlineCourseItem item = new CxgQinOnlineCourseItem();
//                    item.setCourseId(course.getId());
//                    item.setCover(jsonTmp.get("imgPath").toString());
//                    item.setTitle(jsonTmp.get("title").toString());
//
//                    item.setVideoShdUri(getContainsKey(jsonTmp,"mp4ShdUrl"));
//                    item.setVideoShdSize(getContainsKey(jsonTmp,"mp4ShdSize"));
//
//                    item.setVideoHdUri(getContainsKey(jsonTmp,"mp4HdUrl"));
//                    item.setVideoHdSize(getContainsKey(jsonTmp,"mp4HdSize"));
//
//                    item.setVideoSdUri(getContainsKey(jsonTmp,"mp4SdUrl"));
//                    item.setVideoSdSize(getContainsKey(jsonTmp,"mp4SdSize"));
//
//                    item.setSortNum(Convert.toInt(jsonTmp.get("pNumber")));
//
//                    item.setWebUri(jsonTmp.get("webUrl").toString());
//                    item.setDurationTime(Convert.toInt(jsonTmp.get("mLength")));
//                    item.setVideoType(2);
//                    item.setIsInclude(0);
//                    itemList.add(item);
//                }
//                courseItemService.saveBatch(itemList);
//            }
//        }
//        return bookEntity;
//    }
//
//    public String getContainsKey(JSONObject obj, String keyField) {
//
//        String[] keys = Convert.toStrArray(obj.keySet());
//        for (String keyStr : keys) {
//            if (keyStr.contains(keyField) ) {
//                if (StrUtil.isNotEmpty(obj.getString(keyStr))) {
//                    if (keyField.contains("Size")) {
//                        if (obj.getInteger(keyStr) > 0) {
//                            return obj.getString(keyStr);
//                        }
//                    }else {
//                        return obj.getString(keyStr);
//                    }
//                }
//            }
//        }
//        return "";
//    }
//
//    /**
//     * 判断一个字符串是否都为数字
//     *
//     * @param strNum
//     * @return
//     */
//    public boolean isDigit(String strNum) {
//        Pattern pattern = Pattern.compile("[0-9]{1,}");
//        Matcher matcher = pattern.matcher((CharSequence) strNum);
//        return matcher.matches();
//    }
//
//    /**
//     * 截取数字
//     *
//     * @param content
//     * @return
//     */
//    public String getNumbers(String content) {
//        Pattern pattern = Pattern.compile("\\d+");
//        Matcher matcher = pattern.matcher(content);
//        while (matcher.find()) {
//            return matcher.group(0);
//        }
//        return "";
//    }
//}
